// Copyright 2006 PR1ME. All Rights Reserved.
import java.io.IOException;
import java.io.InputStream;

/**
 * Breaks an input stream into a list of words.
 */
public class WordLexer {

  /**
   * A buffer size to use to avoid repeated system calls. We use this instead of
   * BufferedInputStream to avoid the cost of unnecessary synchronizations.
   * Experimentation on the target system suggests this is an optimal buffer
   * size to use. Larger buffers made no positive impact on performance.
   */
  private static final int BUF_SIZE = 8192;

  /**
   * Constructs a word lexer with the specified character analyzer.
   */
  public WordLexer(CharacterAnalyzer cAnalyzer) {
    this.cAnalyzer = cAnalyzer;
  }

  /**
   * Break an input stream into a list of words. Each word of the input is
   * passed to the specified parser's process method in the order in which it
   * appears in the input.
   * 
   * @param in the stream to read from
   * @param parser receives each word found in the input
   * @throws IOException if the stream throws an exception
   */
  public void processInput(InputStream in, WordParser parser)
      throws IOException {
    // Collects characters into the next word to process
    StringBuffer collectWord = new StringBuffer();
    // Start assuming break sequence
    boolean inWordSequence = false;
    // Track line count and total bytes; lines count from 0
    int totalBytes = 0;
    // Read the input stream in chunks to avoid tons of slow IO calls
    int readCount;
    byte[] buf = new byte[BUF_SIZE];
    while ((readCount = in.read(buf, 0, BUF_SIZE)) >= 0) {
      totalBytes += readCount;
      // Process all the bytes we've read so far
      for (int i = 0; i < readCount; ++i) {
        // force to positive int in range 0 to 255
        int ch = buf[i] & 0xFF;
        boolean isBreakChar = cAnalyzer.isBreakChar(ch);
        if (isBreakChar) {
          if (inWordSequence) {
            // Transition from non-break to break; process the collected word
            String lastWord = collectWord.toString();
            // Clear the collected buffer
            collectWord = new StringBuffer();
            parser.processWord(lastWord);
            inWordSequence = false;
          }
        } else {
          // collect the character into the current word
          inWordSequence = true;
          collectWord.append((char) ch);
        }
      }
    }
    // end of input; if we've collected any characters, send the final word
    if (inWordSequence) {
      parser.processWord(collectWord.toString());
    }

    parser.endOfStream(totalBytes);
  }

  /**
   * The character analyzer that determines what a word is.
   */
  private final CharacterAnalyzer cAnalyzer;

}
